<!DOCTYPE html>
<html>
 <head>
  <meta charset="utf-8"/>
  <meta content="width=device-width,initial-scale=1" name="viewport"/>
  <meta content="ie=edge" http-equiv="x-ua-compatible"/>
  <meta content="Copy to clipboard" name="lang:clipboard.copy"/>
  <meta content="Copied to clipboard" name="lang:clipboard.copied"/>
  <meta content="en" name="lang:search.language"/>
  <meta content="True" name="lang:search.pipeline.stopwords"/>
  <meta content="True" name="lang:search.pipeline.trimmer"/>
  <meta content="No matching documents" name="lang:search.result.none"/>
  <meta content="1 matching document" name="lang:search.result.one"/>
  <meta content="# matching documents" name="lang:search.result.other"/>
  <meta content="[\s\-]+" name="lang:search.tokenizer"/>
  <link crossorigin="" href="https://fonts.gstatic.com/" rel="preconnect"/>
  <link href="https://fonts.googleapis.com/css?family=Roboto+Mono:400,500,700|Roboto:300,400,400i,700&amp;display=fallback" rel="stylesheet"/>
  <style>
   body,
      input {
        font-family: "Roboto", "Helvetica Neue", Helvetica, Arial, sans-serif
      }

      code,
      kbd,
      pre {
        font-family: "Roboto Mono", "Courier New", Courier, monospace
      }
  </style>
  <link href="../_static/stylesheets/application.css" rel="stylesheet"/>
  <link href="../_static/stylesheets/application-palette.css" rel="stylesheet"/>
  <link href="../_static/stylesheets/application-fixes.css" rel="stylesheet"/>
  <link href="../_static/fonts/material-icons.css" rel="stylesheet"/>
  <meta content="84bd00" name="theme-color"/>
  <script src="../_static/javascripts/modernizr.js">
  </script>
  <title>
   Getting Started — TRTorch 0.0.1 documentation
  </title>
  <link href="../_static/material.css" rel="stylesheet" type="text/css"/>
  <link href="../_static/pygments.css" rel="stylesheet" type="text/css"/>
  <link href="../_static/collapsible-lists/css/tree_view.css" rel="stylesheet" type="text/css"/>
  <script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js">
  </script>
  <script src="../_static/jquery.js">
  </script>
  <script src="../_static/underscore.js">
  </script>
  <script src="../_static/doctools.js">
  </script>
  <script src="../_static/language_data.js">
  </script>
  <script src="../_static/collapsible-lists/js/CollapsibleLists.compressed.js">
  </script>
  <script src="../_static/collapsible-lists/js/apply-collapsible-lists.js">
  </script>
  <link href="../genindex.html" rel="index" title="Index"/>
  <link href="../search.html" rel="search" title="Search"/>
  <link href="ptq.html" rel="next" title="Post Training Quantization (PTQ)"/>
  <link href="installation.html" rel="prev" title="Installation"/>
 </head>
 <body data-md-color-accent="light-green" data-md-color-primary="light-green" dir="ltr">
  <svg class="md-svg">
   <defs data-children-count="0">
    <svg height="448" id="__github" viewbox="0 0 416 448" width="416" xmlns="http://www.w3.org/2000/svg">
     <path d="M160 304q0 10-3.125 20.5t-10.75 19T128 352t-18.125-8.5-10.75-19T96 304t3.125-20.5 10.75-19T128 256t18.125 8.5 10.75 19T160 304zm160 0q0 10-3.125 20.5t-10.75 19T288 352t-18.125-8.5-10.75-19T256 304t3.125-20.5 10.75-19T288 256t18.125 8.5 10.75 19T320 304zm40 0q0-30-17.25-51T296 232q-10.25 0-48.75 5.25Q229.5 240 208 240t-39.25-2.75Q130.75 232 120 232q-29.5 0-46.75 21T56 304q0 22 8 38.375t20.25 25.75 30.5 15 35 7.375 37.25 1.75h42q20.5 0 37.25-1.75t35-7.375 30.5-15 20.25-25.75T360 304zm56-44q0 51.75-15.25 82.75-9.5 19.25-26.375 33.25t-35.25 21.5-42.5 11.875-42.875 5.5T212 416q-19.5 0-35.5-.75t-36.875-3.125-38.125-7.5-34.25-12.875T37 371.5t-21.5-28.75Q0 312 0 260q0-59.25 34-99-6.75-20.5-6.75-42.5 0-29 12.75-54.5 27 0 47.5 9.875t47.25 30.875Q171.5 96 212 96q37 0 70 8 26.25-20.5 46.75-30.25T376 64q12.75 25.5 12.75 54.5 0 21.75-6.75 42 34 40 34 99.5z" fill="currentColor">
     </path>
    </svg>
   </defs>
  </svg>
  <input class="md-toggle" data-md-toggle="drawer" id="__drawer" type="checkbox"/>
  <input class="md-toggle" data-md-toggle="search" id="__search" type="checkbox"/>
  <label class="md-overlay" data-md-component="overlay" for="__drawer">
  </label>
  <a class="md-skip" href="#tutorials/getting_started" tabindex="1">
   Skip to content
  </a>
  <header class="md-header" data-md-component="header">
   <nav class="md-header-nav md-grid">
    <div class="md-flex navheader">
     <div class="md-flex__cell md-flex__cell--shrink">
      <a class="md-header-nav__button md-logo" href="../index.html" title="TRTorch 0.0.1 documentation">
       <i class="md-icon">
        
       </i>
      </a>
     </div>
     <div class="md-flex__cell md-flex__cell--shrink">
      <label class="md-icon md-icon--menu md-header-nav__button" for="__drawer">
      </label>
     </div>
     <div class="md-flex__cell md-flex__cell--stretch">
      <div class="md-flex__ellipsis md-header-nav__title" data-md-component="title">
       <span class="md-header-nav__topic">
        TRTorch
       </span>
       <span class="md-header-nav__topic">
        Getting Started
       </span>
      </div>
     </div>
     <div class="md-flex__cell md-flex__cell--shrink">
      <label class="md-icon md-icon--search md-header-nav__button" for="__search">
      </label>
      <div class="md-search" data-md-component="search" role="dialog">
       <label class="md-search__overlay" for="__search">
       </label>
       <div class="md-search__inner" role="search">
        <form action="../search.html" class="md-search__form" method="GET" name="search">
         <input autocapitalize="off" autocomplete="off" class="md-search__input" data-md-component="query" data-md-state="active" name="q" placeholder="Search" spellcheck="false" type="text"/>
         <label class="md-icon md-search__icon" for="__search">
         </label>
         <button class="md-icon md-search__icon" data-md-component="reset" tabindex="-1" type="reset">
          
         </button>
        </form>
        <div class="md-search__output">
         <div class="md-search__scrollwrap" data-md-scrollfix="">
          <div class="md-search-result" data-md-component="result">
           <div class="md-search-result__meta">
            Type to start searching
           </div>
           <ol class="md-search-result__list">
           </ol>
          </div>
         </div>
        </div>
       </div>
      </div>
     </div>
     <div class="md-flex__cell md-flex__cell--shrink">
      <div class="md-header-nav__source">
       <a class="md-source" data-md-source="github" href="https://github.com/nvidia/TRTorch/" title="Go to repository">
        <div class="md-source__icon">
         <svg height="28" viewbox="0 0 24 24" width="28" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
          <use height="24" width="24" xlink:href="#__github">
          </use>
         </svg>
        </div>
        <div class="md-source__repository">
         TRTorch
        </div>
       </a>
      </div>
     </div>
     <div class="md-flex__cell md-flex__cell--shrink dropdown">
      <button class="dropdownbutton">
       Versions
      </button>
      <div class="dropdown-content md-hero">
       <a href="https://nvidia.github.io/TRTorch/" title="master">
        master
       </a>
       <a href="https://nvidia.github.io/TRTorch/v0.0.1/" title="v0.0.1">
        v0.0.1
       </a>
      </div>
     </div>
    </div>
   </nav>
  </header>
  <div class="md-container">
   <nav class="md-tabs" data-md-component="tabs">
    <div class="md-tabs__inner md-grid">
     <ul class="md-tabs__list">
      <li class="md-tabs__item">
       <a class="md-tabs__link" href="../index.html">
        TRTorch 0.0.1 documentation
       </a>
      </li>
     </ul>
    </div>
   </nav>
   <main class="md-main">
    <div class="md-main__inner md-grid" data-md-component="container">
     <div class="md-sidebar md-sidebar--primary" data-md-component="navigation">
      <div class="md-sidebar__scrollwrap">
       <div class="md-sidebar__inner">
        <nav class="md-nav md-nav--primary" data-md-level="0">
         <label class="md-nav__title md-nav__title--site" for="__drawer">
          <a class="md-nav__button md-logo" href="../index.html" title="TRTorch 0.0.1 documentation">
           <i class="md-icon">
            
           </i>
          </a>
          <a href="../index.html" title="TRTorch 0.0.1 documentation">
           TRTorch
          </a>
         </label>
         <div class="md-nav__source">
          <a class="md-source" data-md-source="github" href="https://github.com/nvidia/TRTorch/" title="Go to repository">
           <div class="md-source__icon">
            <svg height="28" viewbox="0 0 24 24" width="28" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
             <use height="24" width="24" xlink:href="#__github">
             </use>
            </svg>
           </div>
           <div class="md-source__repository">
            TRTorch
           </div>
          </a>
         </div>
         <ul class="md-nav__list">
          <li class="md-nav__item">
           <span class="md-nav__link caption">
            <span class="caption-text">
             Getting Started
            </span>
           </span>
          </li>
          <li class="md-nav__item">
           <a class="md-nav__link" href="installation.html">
            Installation
           </a>
           <ul class="md-nav__list">
            <li class="md-nav__item">
             <a class="md-nav__link" href="installation.html#binary-distribution">
              Binary Distribution
             </a>
            </li>
            <li class="md-nav__item">
             <a class="md-nav__link" href="installation.html#compiling-from-source">
              Compiling From Source
             </a>
            </li>
           </ul>
          </li>
          <li class="md-nav__item">
           <input class="md-toggle md-nav__toggle" data-md-toggle="toc" id="__toc" type="checkbox"/>
           <label class="md-nav__link md-nav__link--active" for="__toc">
            Getting Started
           </label>
           <a class="md-nav__link md-nav__link--active" href="#">
            Getting Started
           </a>
           <nav class="md-nav md-nav--secondary">
            <label class="md-nav__title" for="__toc">
             Contents
            </label>
            <ul class="md-nav__list" data-md-scrollfix="">
             <li class="md-nav__item">
              <a class="md-nav__link" href="#tutorials-getting-started--page-root">
               Getting Started
              </a>
              <nav class="md-nav">
               <ul class="md-nav__list">
                <li class="md-nav__item">
                 <a class="md-nav__link" href="#creating-a-torchscript-module">
                  Creating a TorchScript Module
                 </a>
                </li>
                <li class="md-nav__item">
                 <a class="md-nav__link" href="#working-with-torchscript-in-c">
                  Working with TorchScript in C++
                 </a>
                </li>
                <li class="md-nav__item">
                 <a class="md-nav__link" href="#compiling-with-trtorch">
                  Compiling with TRTorch
                 </a>
                </li>
                <li class="md-nav__item">
                 <a class="md-nav__link" href="#under-the-hood">
                  Under The Hood
                 </a>
                </li>
                <li class="md-nav__item">
                 <a class="md-nav__link" href="#working-with-unsupported-operators">
                  Working with Unsupported Operators
                 </a>
                 <nav class="md-nav">
                  <ul class="md-nav__list">
                   <li class="md-nav__item">
                    <a class="md-nav__link" href="#registering-custom-converters">
                     Registering Custom Converters
                    </a>
                   </li>
                  </ul>
                 </nav>
                </li>
               </ul>
              </nav>
             </li>
             <li class="md-nav__item">
              <a class="md-nav__extra_link" href="../_sources/tutorials/getting_started.rst.txt">
               Show Source
              </a>
             </li>
            </ul>
           </nav>
           <ul class="md-nav__list">
            <li class="md-nav__item">
             <a class="md-nav__link" href="#creating-a-torchscript-module">
              Creating a TorchScript Module
             </a>
            </li>
            <li class="md-nav__item">
             <a class="md-nav__link" href="#working-with-torchscript-in-c">
              Working with TorchScript in C++
             </a>
            </li>
            <li class="md-nav__item">
             <a class="md-nav__link" href="#compiling-with-trtorch">
              Compiling with TRTorch
             </a>
            </li>
            <li class="md-nav__item">
             <a class="md-nav__link" href="#under-the-hood">
              Under The Hood
             </a>
            </li>
            <li class="md-nav__item">
             <a class="md-nav__link" href="#working-with-unsupported-operators">
              Working with Unsupported Operators
             </a>
            </li>
           </ul>
          </li>
          <li class="md-nav__item">
           <a class="md-nav__link" href="ptq.html">
            Post Training Quantization (PTQ)
           </a>
           <ul class="md-nav__list">
            <li class="md-nav__item">
             <a class="md-nav__link" href="ptq.html#how-to-create-your-own-ptq-application">
              How to create your own PTQ application
             </a>
            </li>
           </ul>
          </li>
          <li class="md-nav__item">
           <span class="md-nav__link caption">
            <span class="caption-text">
             Contributor Documentation
            </span>
           </span>
          </li>
          <li class="md-nav__item">
           <a class="md-nav__link" href="../contributors/system_overview.html">
            System Overview
           </a>
           <ul class="md-nav__list">
            <li class="md-nav__item">
             <a class="md-nav__link" href="../contributors/system_overview.html#compiler-phases">
              Compiler Phases
             </a>
            </li>
           </ul>
          </li>
          <li class="md-nav__item">
           <a class="md-nav__link" href="../contributors/writing_converters.html">
            Writing Converters
           </a>
           <ul class="md-nav__list">
            <li class="md-nav__item">
             <a class="md-nav__link" href="../contributors/writing_converters.html#background">
              Background
             </a>
            </li>
            <li class="md-nav__item">
             <a class="md-nav__link" href="../contributors/writing_converters.html#converters">
              Converters
             </a>
            </li>
            <li class="md-nav__item">
             <a class="md-nav__link" href="../contributors/writing_converters.html#converter-contract">
              Converter Contract
             </a>
            </li>
            <li class="md-nav__item">
             <a class="md-nav__link" href="../contributors/writing_converters.html#conversion-context">
              Conversion Context
             </a>
            </li>
            <li class="md-nav__item">
             <a class="md-nav__link" href="../contributors/writing_converters.html#args">
              Args
             </a>
            </li>
            <li class="md-nav__item">
             <a class="md-nav__link" href="../contributors/writing_converters.html#weights">
              Weights
             </a>
            </li>
            <li class="md-nav__item">
             <a class="md-nav__link" href="../contributors/writing_converters.html#other-advice">
              Other advice
             </a>
            </li>
           </ul>
          </li>
          <li class="md-nav__item">
           <span class="md-nav__link caption">
            <span class="caption-text">
             C++ API Documenation
            </span>
           </span>
          </li>
          <li class="md-nav__item">
           <a class="md-nav__link" href="../_api/trtorch_cpp.html">
            TRTorch C++ API
           </a>
           <ul class="md-nav__list">
            <li class="md-nav__item">
             <a class="md-nav__link" href="../_api/trtorch_cpp.html#class-hierarchy">
              Class Hierarchy
             </a>
            </li>
            <li class="md-nav__item">
             <a class="md-nav__link" href="../_api/trtorch_cpp.html#file-hierarchy">
              File Hierarchy
             </a>
            </li>
            <li class="md-nav__item">
             <a class="md-nav__link" href="../_api/trtorch_cpp.html#full-api">
              Full API
             </a>
            </li>
           </ul>
          </li>
         </ul>
        </nav>
       </div>
      </div>
     </div>
     <div class="md-sidebar md-sidebar--secondary" data-md-component="toc">
      <div class="md-sidebar__scrollwrap">
       <div class="md-sidebar__inner">
        <nav class="md-nav md-nav--secondary">
         <label class="md-nav__title" for="__toc">
          Contents
         </label>
         <ul class="md-nav__list" data-md-scrollfix="">
          <li class="md-nav__item">
           <a class="md-nav__link" href="#tutorials-getting-started--page-root">
            Getting Started
           </a>
           <nav class="md-nav">
            <ul class="md-nav__list">
             <li class="md-nav__item">
              <a class="md-nav__link" href="#creating-a-torchscript-module">
               Creating a TorchScript Module
              </a>
             </li>
             <li class="md-nav__item">
              <a class="md-nav__link" href="#working-with-torchscript-in-c">
               Working with TorchScript in C++
              </a>
             </li>
             <li class="md-nav__item">
              <a class="md-nav__link" href="#compiling-with-trtorch">
               Compiling with TRTorch
              </a>
             </li>
             <li class="md-nav__item">
              <a class="md-nav__link" href="#under-the-hood">
               Under The Hood
              </a>
             </li>
             <li class="md-nav__item">
              <a class="md-nav__link" href="#working-with-unsupported-operators">
               Working with Unsupported Operators
              </a>
              <nav class="md-nav">
               <ul class="md-nav__list">
                <li class="md-nav__item">
                 <a class="md-nav__link" href="#registering-custom-converters">
                  Registering Custom Converters
                 </a>
                </li>
               </ul>
              </nav>
             </li>
            </ul>
           </nav>
          </li>
          <li class="md-nav__item">
           <a class="md-nav__extra_link" href="../_sources/tutorials/getting_started.rst.txt">
            Show Source
           </a>
          </li>
          <li class="md-nav__item" id="searchbox">
          </li>
         </ul>
        </nav>
       </div>
      </div>
     </div>
     <div class="md-content">
      <article class="md-content__inner md-typeset" role="main">
       <span id="id1">
       </span>
       <h1 id="tutorials-getting-started--page-root">
        Getting Started
        <a class="headerlink" href="#tutorials-getting-started--page-root" title="Permalink to this headline">
         ¶
        </a>
       </h1>
       <p>
        If you haven’t already, aquire a tarball of the library by following the instructions in
        <a class="reference internal" href="installation.html#installation">
         <span class="std std-ref">
          Installation
         </span>
        </a>
       </p>
       <h2 id="creating-a-torchscript-module">
        Creating a TorchScript Module
        <a class="headerlink" href="#creating-a-torchscript-module" title="Permalink to this headline">
         ¶
        </a>
       </h2>
       <p>
        Once you have a trained model you want to compile with TRTorch, you need to start by converting that model from Python code to TorchScript code.
PyTorch has detailed documentation on how to do this
        <a class="reference external" href="https://pytorch.org/tutorials/beginner/Intro_to_TorchScript_tutorial.html">
         https://pytorch.org/tutorials/beginner/Intro_to_TorchScript_tutorial.html
        </a>
        but briefly here is the
here is key background and the process:
       </p>
       <p>
        PyTorch programs are based around
        <a href="#id2">
         <span class="problematic" id="id3">
          `
         </span>
        </a>
        Module`s which can be used to compose higher level modules. Modules contain a constructor to set up the modules, parameters and sub-modules
and a forward function which describes how to use the parameters and submodules when the module is invoked.
       </p>
       <p>
        For example, we can define a LeNet module like this:
       </p>
       <div class="highlight-python notranslate">
        <table class="highlighttable">
         <tr>
          <td class="linenos">
           <div class="linenodiv">
            <pre> 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35</pre>
           </div>
          </td>
          <td class="code">
           <div class="highlight">
            <pre><span></span><span class="k">class</span> <span class="nc">LeNetFeatExtractor</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">LeNetFeatExtractor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">conv1</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Conv2d</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">6</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">conv2</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Conv2d</span><span class="p">(</span><span class="mi">6</span><span class="p">,</span> <span class="mi">16</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
        <span class="n">x</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">max_pool2d</span><span class="p">(</span><span class="n">F</span><span class="o">.</span><span class="n">relu</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">conv1</span><span class="p">(</span><span class="n">x</span><span class="p">)),</span> <span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">))</span>
        <span class="n">x</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">max_pool2d</span><span class="p">(</span><span class="n">F</span><span class="o">.</span><span class="n">relu</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">conv2</span><span class="p">(</span><span class="n">x</span><span class="p">)),</span> <span class="mi">2</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">x</span>

<span class="k">class</span> <span class="nc">LeNetClassifier</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">LeNetClassifier</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">fc1</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="mi">16</span> <span class="o">*</span> <span class="mi">6</span> <span class="o">*</span> <span class="mi">6</span><span class="p">,</span> <span class="mi">120</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">fc2</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="mi">120</span><span class="p">,</span> <span class="mi">84</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">fc3</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="mi">84</span><span class="p">,</span> <span class="mi">10</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
        <span class="n">x</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">flatten</span><span class="p">(</span><span class="n">x</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
        <span class="n">x</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">relu</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">fc1</span><span class="p">(</span><span class="n">x</span><span class="p">))</span>
        <span class="n">x</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">relu</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">fc2</span><span class="p">(</span><span class="n">x</span><span class="p">))</span>
        <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">fc3</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">x</span>

<span class="k">class</span> <span class="nc">LeNet</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">LeNet</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">feat</span> <span class="o">=</span> <span class="n">LeNetFeatExtractor</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">classifer</span> <span class="o">=</span> <span class="n">LeNetClassifier</span><span class="p">()</span>

    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
        <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">feat</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
        <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">classifer</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">x</span>
</pre>
           </div>
          </td>
         </tr>
        </table>
       </div>
       <p>
        .
       </p>
       <blockquote>
        <div>
         <p>
          Obviously you may want to consolidate such a simple model into a single module but we can see the composability of PyTorch here
         </p>
        </div>
       </blockquote>
       <p>
        From here are two pathways for going from PyTorch Python code to TorchScript code: Tracing and Scripting.
       </p>
       <p>
        Tracing follows the path of execution when the module is called and records what happens.
To trace an instance of our LeNet module, we can call
        <code class="docutils literal notranslate">
         <span class="pre">
          torch.jit.trace
         </span>
        </code>
        with an example input.
       </p>
       <div class="highlight-python notranslate">
        <div class="highlight">
         <pre><span></span><span class="n">model</span> <span class="o">=</span> <span class="n">LeNet</span><span class="p">()</span>
<span class="n">traced_model</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">jit</span><span class="o">.</span><span class="n">trace</span><span class="p">(</span><span class="n">model</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">([</span><span class="mi">1</span><span class="p">,</span><span class="mi">1</span><span class="p">,</span><span class="mi">32</span><span class="p">,</span><span class="mi">32</span><span class="p">]))</span>
</pre>
        </div>
       </div>
       <p>
        Scripting actually inspects your code with a compiler and generates an equivalent TorchScript program. The difference is that since tracing
is following the execution of your module, it cannot pick up control flow for instance. By working from the Python code, the compiler can
include these components. We can run the script compiler on our LeNet module by calling
        <code class="docutils literal notranslate">
         <span class="pre">
          torch.jit.script
         </span>
        </code>
       </p>
       <div class="highlight-python notranslate">
        <div class="highlight">
         <pre><span></span><span class="n">model</span> <span class="o">=</span> <span class="n">LeNet</span><span class="p">()</span>
<span class="n">script_model</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">jit</span><span class="o">.</span><span class="n">script</span><span class="p">(</span><span class="n">model</span><span class="p">)</span>
</pre>
        </div>
       </div>
       <p>
        There are reasons to use one path or another, the PyTorch documentation has information on how to choose. From a TRTorch prespective, there is
better support (i.e your module is more likely to compile) for traced modules because it doesn’t include all the complexities of a complete
programming language, though both paths supported.
       </p>
       <p>
        After scripting or tracing your module, you are given back a TorchScript Module. This contains the code and parameters used to run the module stored
in a intermediate representation that TRTorch can consume.
       </p>
       <p>
        Here is what the LeNet traced module IR looks like:
       </p>
       <div class="highlight-none notranslate">
        <div class="highlight">
         <pre><span></span>graph(%self.1 : __torch__.___torch_mangle_10.LeNet,
    %input.1 : Float(1, 1, 32, 32)):
    %129 : __torch__.___torch_mangle_9.LeNetClassifier = prim::GetAttr[name="classifer"](%self.1)
    %119 : __torch__.___torch_mangle_5.LeNetFeatExtractor = prim::GetAttr[name="feat"](%self.1)
    %137 : Tensor = prim::CallMethod[name="forward"](%119, %input.1)
    %138 : Tensor = prim::CallMethod[name="forward"](%129, %137)
    return (%138)
</pre>
        </div>
       </div>
       <p>
        and the LeNet scripted module IR:
       </p>
       <div class="highlight-none notranslate">
        <div class="highlight">
         <pre><span></span>graph(%self : __torch__.LeNet,
    %x.1 : Tensor):
    %2 : __torch__.LeNetFeatExtractor = prim::GetAttr[name="feat"](%self)
    %x.3 : Tensor = prim::CallMethod[name="forward"](%2, %x.1) # x.py:38:12
    %5 : __torch__.LeNetClassifier = prim::GetAttr[name="classifer"](%self)
    %x.5 : Tensor = prim::CallMethod[name="forward"](%5, %x.3) # x.py:39:12
    return (%x.5)
</pre>
        </div>
       </div>
       <p>
        You can see that the IR preserves the module structure we have in our python code.
       </p>
       <p>
        From here we can save either our traced or scripted module using
        <code class="docutils literal notranslate">
         <span class="pre">
          torch.jit.save
         </span>
        </code>
        which will serialize the TorchScript code, weights and other information into a package.
This is also where our dependency on Python ends.
       </p>
       <div class="highlight-python notranslate">
        <div class="highlight">
         <pre><span></span><span class="n">torch_script_module</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="s2">"lenet.jit.pt"</span><span class="p">)</span>
</pre>
        </div>
       </div>
       <span id="ts-in-cc">
       </span>
       <h2 id="working-with-torchscript-in-c">
        Working with TorchScript in C++
        <a class="headerlink" href="#working-with-torchscript-in-c" title="Permalink to this headline">
         ¶
        </a>
       </h2>
       <p>
        From here we can now load our TorchScript module in C++
       </p>
       <div class="highlight-c++ notranslate">
        <div class="highlight">
         <pre><span></span><span class="cp">#include</span> <span class="cpf">&lt;torch/script.h&gt; // One-stop header.</span><span class="cp"></span>

<span class="cp">#include</span> <span class="cpf">&lt;iostream&gt;</span><span class="cp"></span>
<span class="cp">#include</span> <span class="cpf">&lt;memory&gt;</span><span class="cp"></span>

<span class="kt">int</span> <span class="nf">main</span><span class="p">(</span><span class="kt">int</span> <span class="n">argc</span><span class="p">,</span> <span class="k">const</span> <span class="kt">char</span><span class="o">*</span> <span class="n">argv</span><span class="p">[])</span> <span class="p">{</span>
    <span class="n">torch</span><span class="o">::</span><span class="n">jit</span><span class="o">::</span><span class="n">Module</span> <span class="n">module</span><span class="p">;</span>
    <span class="k">try</span> <span class="p">{</span>
        <span class="c1">// Deserialize the ScriptModule from a file using torch::jit::load().</span>
        <span class="n">module</span> <span class="o">=</span> <span class="n">torch</span><span class="o">::</span><span class="n">jit</span><span class="o">::</span><span class="n">load</span><span class="p">(</span><span class="s">"&lt;PATH TO SAVED TS MOD&gt;"</span><span class="p">);</span>
    <span class="p">}</span>
    <span class="k">catch</span> <span class="p">(</span><span class="k">const</span> <span class="n">c10</span><span class="o">::</span><span class="n">Error</span><span class="o">&amp;</span> <span class="n">e</span><span class="p">)</span> <span class="p">{</span>
        <span class="n">std</span><span class="o">::</span><span class="n">cerr</span> <span class="o">&lt;&lt;</span> <span class="s">"error loading the model</span><span class="se">\n</span><span class="s">"</span><span class="p">;</span>
        <span class="k">return</span> <span class="o">-</span><span class="mi">1</span><span class="p">;</span>
    <span class="p">}</span>

    <span class="n">std</span><span class="o">::</span><span class="n">cout</span> <span class="o">&lt;&lt;</span> <span class="s">"ok</span><span class="se">\n</span><span class="s">"</span><span class="p">;</span>
</pre>
        </div>
       </div>
       <p>
        You can do full training and inference in C++ with PyTorch / LibTorch if you would like, you can even define your modules in C++ and
have access to the same powerful tensor library that backs PyTorch. (For more information:
        <a class="reference external" href="https://pytorch.org/cppdocs/">
         https://pytorch.org/cppdocs/
        </a>
        ).
For instance we can do inference with our LeNet module like this:
       </p>
       <div class="highlight-c++ notranslate">
        <div class="highlight">
         <pre><span></span><span class="n">mod</span><span class="p">.</span><span class="n">eval</span><span class="p">();</span>
<span class="n">torch</span><span class="o">::</span><span class="n">Tensor</span> <span class="n">in</span> <span class="o">=</span> <span class="n">torch</span><span class="o">::</span><span class="n">randn</span><span class="p">({</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">32</span><span class="p">,</span> <span class="mi">32</span><span class="p">});</span>
<span class="k">auto</span> <span class="n">out</span> <span class="o">=</span> <span class="n">mod</span><span class="p">.</span><span class="n">forward</span><span class="p">(</span><span class="n">in</span><span class="p">);</span>
</pre>
        </div>
       </div>
       <p>
        and to run on the GPU:
       </p>
       <div class="highlight-c++ notranslate">
        <div class="highlight">
         <pre><span></span><span class="n">mod</span><span class="p">.</span><span class="n">eval</span><span class="p">();</span>
<span class="n">mod</span><span class="p">.</span><span class="n">to</span><span class="p">(</span><span class="n">torch</span><span class="o">::</span><span class="n">kCUDA</span><span class="p">);</span>
<span class="n">torch</span><span class="o">::</span><span class="n">Tensor</span> <span class="n">in</span> <span class="o">=</span> <span class="n">torch</span><span class="o">::</span><span class="n">randn</span><span class="p">({</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">32</span><span class="p">,</span> <span class="mi">32</span><span class="p">},</span> <span class="n">torch</span><span class="o">::</span><span class="n">kCUDA</span><span class="p">);</span>
<span class="k">auto</span> <span class="n">out</span> <span class="o">=</span> <span class="n">mod</span><span class="p">.</span><span class="n">forward</span><span class="p">(</span><span class="n">in</span><span class="p">);</span>
</pre>
        </div>
       </div>
       <p>
        As you can see it is pretty similar to the Python API. When you call the
        <code class="docutils literal notranslate">
         <span class="pre">
          forward
         </span>
        </code>
        method, you invoke the PyTorch JIT compiler, which will optimize and run your TorchScript code.
       </p>
       <span id="compile">
       </span>
       <h2 id="compiling-with-trtorch">
        Compiling with TRTorch
        <a class="headerlink" href="#compiling-with-trtorch" title="Permalink to this headline">
         ¶
        </a>
       </h2>
       <p>
        We are also at the point were we can compile and optimize our module with TRTorch, but instead of in a JIT fashion we must do it ahead-of-time (AOT) i.e. before we start doing actual inference work
since it takes a bit of time to optimize the module, it would not make sense to do this every time you run the module or even the first time you run it.
       </p>
       <p>
        With out module loaded, we can feed it into the TRTorch compiler. When we do so we must provide some information on the expected input size and also configure any additional settings.
       </p>
       <div class="highlight-c++ notranslate">
        <div class="highlight">
         <pre><span></span><span class="cp">#include</span> <span class="cpf">"torch/script.h"</span><span class="cp"></span>
<span class="cp">#include</span> <span class="cpf">"trtorch/trtorch.h"</span><span class="cp"></span>
<span class="p">...</span>

    <span class="n">mod</span><span class="p">.</span><span class="n">to</span><span class="p">(</span><span class="n">at</span><span class="o">::</span><span class="n">kCUDA</span><span class="p">);</span>
    <span class="n">mod</span><span class="p">.</span><span class="n">eval</span><span class="p">();</span>

    <span class="k">auto</span> <span class="n">in</span> <span class="o">=</span> <span class="n">torch</span><span class="o">::</span><span class="n">randn</span><span class="p">({</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">32</span><span class="p">,</span> <span class="mi">32</span><span class="p">},</span> <span class="p">{</span><span class="n">torch</span><span class="o">::</span><span class="n">kCUDA</span><span class="p">});</span>
    <span class="k">auto</span> <span class="n">trt_mod</span> <span class="o">=</span> <span class="n">trtorch</span><span class="o">::</span><span class="n">CompileGraph</span><span class="p">(</span><span class="n">mod</span><span class="p">,</span> <span class="n">std</span><span class="o">::</span><span class="n">vector</span><span class="o">&lt;</span><span class="n">trtorch</span><span class="o">::</span><span class="n">ExtraInfo</span><span class="o">::</span><span class="n">InputRange</span><span class="o">&gt;</span><span class="p">{{</span><span class="n">in</span><span class="p">.</span><span class="n">sizes</span><span class="p">()}});</span>
    <span class="k">auto</span> <span class="n">out</span> <span class="o">=</span> <span class="n">trt_mod</span><span class="p">.</span><span class="n">forward</span><span class="p">({</span><span class="n">in</span><span class="p">});</span>
</pre>
        </div>
       </div>
       <p>
        Thats it! Now the graph runs primarily not with the JIT compiler but using TensorRT (though we execute the graph using the JIT runtime).
       </p>
       <p>
        We can also set settings like operating precision to run in FP16.
       </p>
       <div class="highlight-c++ notranslate">
        <div class="highlight">
         <pre><span></span><span class="cp">#include</span> <span class="cpf">"torch/script.h"</span><span class="cp"></span>
<span class="cp">#include</span> <span class="cpf">"trtorch/trtorch.h"</span><span class="cp"></span>
<span class="p">...</span>

    <span class="n">mod</span><span class="p">.</span><span class="n">to</span><span class="p">(</span><span class="n">at</span><span class="o">::</span><span class="n">kCUDA</span><span class="p">);</span>
    <span class="n">mod</span><span class="p">.</span><span class="n">eval</span><span class="p">();</span>

    <span class="k">auto</span> <span class="n">in</span> <span class="o">=</span> <span class="n">torch</span><span class="o">::</span><span class="n">randn</span><span class="p">({</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">32</span><span class="p">,</span> <span class="mi">32</span><span class="p">},</span> <span class="p">{</span><span class="n">torch</span><span class="o">::</span><span class="n">kCUDA</span><span class="p">}).</span><span class="n">to</span><span class="p">(</span><span class="n">torch</span><span class="o">::</span><span class="n">kHALF</span><span class="p">);</span>
    <span class="k">auto</span> <span class="n">input_sizes</span> <span class="o">=</span> <span class="n">std</span><span class="o">::</span><span class="n">vector</span><span class="o">&lt;</span><span class="n">trtorch</span><span class="o">::</span><span class="n">ExtraInfo</span><span class="o">::</span><span class="n">InputRange</span><span class="o">&gt;</span><span class="p">({</span><span class="n">in</span><span class="p">.</span><span class="n">sizes</span><span class="p">()});</span>
    <span class="n">trtorch</span><span class="o">::</span><span class="n">ExtraInfo</span> <span class="n">info</span><span class="p">(</span><span class="n">input_sizes</span><span class="p">);</span>
    <span class="n">info</span><span class="p">.</span><span class="n">op_precision</span> <span class="o">=</span> <span class="n">torch</span><span class="o">::</span><span class="n">kHALF</span><span class="p">;</span>
    <span class="k">auto</span> <span class="n">trt_mod</span> <span class="o">=</span> <span class="n">trtorch</span><span class="o">::</span><span class="n">CompileGraph</span><span class="p">(</span><span class="n">mod</span><span class="p">,</span> <span class="n">info</span><span class="p">);</span>
    <span class="k">auto</span> <span class="n">out</span> <span class="o">=</span> <span class="n">trt_mod</span><span class="p">.</span><span class="n">forward</span><span class="p">({</span><span class="n">in</span><span class="p">});</span>
</pre>
        </div>
       </div>
       <p>
        And now we are running the module in FP16 precision.
       </p>
       <p>
        If you want to save the engine produced by TRTorch to use in a TensorRT application you can use the
        <code class="docutils literal notranslate">
         <span class="pre">
          ConvertGraphToTRTEngine
         </span>
        </code>
        API.
       </p>
       <div class="highlight-c++ notranslate">
        <div class="highlight">
         <pre><span></span><span class="cp">#include</span> <span class="cpf">"torch/script.h"</span><span class="cp"></span>
<span class="cp">#include</span> <span class="cpf">"trtorch/trtorch.h"</span><span class="cp"></span>
<span class="p">...</span>

    <span class="n">mod</span><span class="p">.</span><span class="n">to</span><span class="p">(</span><span class="n">at</span><span class="o">::</span><span class="n">kCUDA</span><span class="p">);</span>
    <span class="n">mod</span><span class="p">.</span><span class="n">eval</span><span class="p">();</span>

    <span class="k">auto</span> <span class="n">in</span> <span class="o">=</span> <span class="n">torch</span><span class="o">::</span><span class="n">randn</span><span class="p">({</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">32</span><span class="p">,</span> <span class="mi">32</span><span class="p">},</span> <span class="p">{</span><span class="n">torch</span><span class="o">::</span><span class="n">kCUDA</span><span class="p">}).</span><span class="n">to</span><span class="p">(</span><span class="n">torch</span><span class="o">::</span><span class="n">kHALF</span><span class="p">);</span>
    <span class="k">auto</span> <span class="n">input_sizes</span> <span class="o">=</span> <span class="n">std</span><span class="o">::</span><span class="n">vector</span><span class="o">&lt;</span><span class="n">trtorch</span><span class="o">::</span><span class="n">ExtraInfo</span><span class="o">::</span><span class="n">InputRange</span><span class="o">&gt;</span><span class="p">({</span><span class="n">in</span><span class="p">.</span><span class="n">sizes</span><span class="p">()});</span>
    <span class="n">trtorch</span><span class="o">::</span><span class="n">ExtraInfo</span> <span class="n">info</span><span class="p">(</span><span class="n">input_sizes</span><span class="p">);</span>
    <span class="n">info</span><span class="p">.</span><span class="n">op_precision</span> <span class="o">=</span> <span class="n">torch</span><span class="o">::</span><span class="n">kHALF</span><span class="p">;</span>
    <span class="k">auto</span> <span class="n">trt_mod</span> <span class="o">=</span> <span class="n">trtorch</span><span class="o">::</span><span class="n">ConvertGraphToTRTEngine</span><span class="p">(</span><span class="n">mod</span><span class="p">,</span> <span class="s">"forward"</span><span class="p">,</span> <span class="n">info</span><span class="p">);</span>
    <span class="n">std</span><span class="o">::</span><span class="n">ofstream</span> <span class="n">out</span><span class="p">(</span><span class="s">"/tmp/engine_converted_from_jit.trt"</span><span class="p">);</span>
    <span class="n">out</span> <span class="o">&lt;&lt;</span> <span class="n">engine</span><span class="p">;</span>
    <span class="n">out</span><span class="p">.</span><span class="n">close</span><span class="p">();</span>
</pre>
        </div>
       </div>
       <span id="id4">
       </span>
       <h2 id="under-the-hood">
        Under The Hood
        <a class="headerlink" href="#under-the-hood" title="Permalink to this headline">
         ¶
        </a>
       </h2>
       <p>
        When a module is provided to TRTorch, the compiler starts by mapping a graph like you saw above to a graph like this:
       </p>
       <div class="highlight-none notranslate">
        <div class="highlight">
         <pre><span></span>graph(%input.2 : Tensor):
    %2 : Float(84, 10) = prim::Constant[value=&lt;Tensor&gt;]()
    %3 : Float(120, 84) = prim::Constant[value=&lt;Tensor&gt;]()
    %4 : Float(576, 120) = prim::Constant[value=&lt;Tensor&gt;]()
    %5 : int = prim::Constant[value=-1]() # x.py:25:0
    %6 : int[] = prim::Constant[value=annotate(List[int], [])]()
    %7 : int[] = prim::Constant[value=[2, 2]]()
    %8 : int[] = prim::Constant[value=[0, 0]]()
    %9 : int[] = prim::Constant[value=[1, 1]]()
    %10 : bool = prim::Constant[value=1]() # ~/.local/lib/python3.6/site-packages/torch/nn/modules/conv.py:346:0
    %11 : int = prim::Constant[value=1]() # ~/.local/lib/python3.6/site-packages/torch/nn/functional.py:539:0
    %12 : bool = prim::Constant[value=0]() # ~/.local/lib/python3.6/site-packages/torch/nn/functional.py:539:0
    %self.classifer.fc3.bias : Float(10) = prim::Constant[value= 0.0464  0.0383  0.0678  0.0932  0.1045 -0.0805 -0.0435 -0.0818  0.0208 -0.0358 [ CUDAFloatType{10} ]]()
    %self.classifer.fc2.bias : Float(84) = prim::Constant[value=&lt;Tensor&gt;]()
    %self.classifer.fc1.bias : Float(120) = prim::Constant[value=&lt;Tensor&gt;]()
    %self.feat.conv2.weight : Float(16, 6, 3, 3) = prim::Constant[value=&lt;Tensor&gt;]()
    %self.feat.conv2.bias : Float(16) = prim::Constant[value=&lt;Tensor&gt;]()
    %self.feat.conv1.weight : Float(6, 1, 3, 3) = prim::Constant[value=&lt;Tensor&gt;]()
    %self.feat.conv1.bias : Float(6) = prim::Constant[value= 0.0530 -0.1691  0.2802  0.1502  0.1056 -0.1549 [ CUDAFloatType{6} ]]()
    %input0.4 : Tensor = aten::_convolution(%input.2, %self.feat.conv1.weight, %self.feat.conv1.bias, %9, %8, %9, %12, %8, %11, %12, %12, %10) # ~/.local/lib/python3.6/site-packages/torch/nn/modules/conv.py:346:0
    %input0.5 : Tensor = aten::relu(%input0.4) # ~/.local/lib/python3.6/site-packages/torch/nn/functional.py:1063:0
    %input1.2 : Tensor = aten::max_pool2d(%input0.5, %7, %6, %8, %9, %12) # ~/.local/lib/python3.6/site-packages/torch/nn/functional.py:539:0
    %input0.6 : Tensor = aten::_convolution(%input1.2, %self.feat.conv2.weight, %self.feat.conv2.bias, %9, %8, %9, %12, %8, %11, %12, %12, %10) # ~/.local/lib/python3.6/site-packages/torch/nn/modules/conv.py:346:0
    %input2.1 : Tensor = aten::relu(%input0.6) # ~/.local/lib/python3.6/site-packages/torch/nn/functional.py:1063:0
    %x.1 : Tensor = aten::max_pool2d(%input2.1, %7, %6, %8, %9, %12) # ~/.local/lib/python3.6/site-packages/torch/nn/functional.py:539:0
    %input.1 : Tensor = aten::flatten(%x.1, %11, %5) # x.py:25:0
    %27 : Tensor = aten::matmul(%input.1, %4)
    %28 : Tensor = trt::const(%self.classifer.fc1.bias)
    %29 : Tensor = aten::add_(%28, %27, %11)
    %input0.2 : Tensor = aten::relu(%29) # ~/.local/lib/python3.6/site-packages/torch/nn/functional.py:1063:0
    %31 : Tensor = aten::matmul(%input0.2, %3)
    %32 : Tensor = trt::const(%self.classifer.fc2.bias)
    %33 : Tensor = aten::add_(%32, %31, %11)
    %input1.1 : Tensor = aten::relu(%33) # ~/.local/lib/python3.6/site-packages/torch/nn/functional.py:1063:0
    %35 : Tensor = aten::matmul(%input1.1, %2)
    %36 : Tensor = trt::const(%self.classifer.fc3.bias)
    %37 : Tensor = aten::add_(%36, %35, %11)
    return (%37)
(CompileGraph)
</pre>
        </div>
       </div>
       <p>
        The graph has now been transformed from a collection of modules, each managing their own parameters into a single graph with the parameters inlined
into the graph and all of the operations laid out. TRTorch has also executed a number of optimizations and mappings to make the graph easier to translate to TensorRT.
From here the compiler can assemble the TensorRT engine by following the dataflow through the graph.
       </p>
       <p>
        When the graph construction phase is complete, TRTorch produces a serialized TensorRT engine. From here depending on the API, this engine is returned
to the user or moves into the graph construction phase. Here TRTorch creates a JIT Module to execute the TensorRT engine which will be instantiated and managed
by the TRTorch runtime.
       </p>
       <p>
        Here is the graph that you get back after compilation is complete:
       </p>
       <div class="highlight-none notranslate">
        <div class="highlight">
         <pre><span></span>graph(%self.1 : __torch__.___torch_mangle_10.LeNet_trt,
    %2 : Tensor):
    %1 : int = prim::Constant[value=94106001690080]()
    %3 : Tensor = trt::execute_engine(%1, %2)
    return (%3)
(AddEngineToGraph)
</pre>
        </div>
       </div>
       <p>
        You can see the call where the engine is executed, based on a constant which is the ID of the engine, telling JIT how to find the engine and the input tensor which will be fed to TensorRT.
       </p>
       <span id="unsupported-ops">
       </span>
       <h2 id="working-with-unsupported-operators">
        Working with Unsupported Operators
        <a class="headerlink" href="#working-with-unsupported-operators" title="Permalink to this headline">
         ¶
        </a>
       </h2>
       <p>
        TRTorch is a new library and the PyTorch operator library is quite large, so there will be ops that aren’t supported natively by the compiler. You can either use the composition techinques
shown above to make modules are fully TRTorch supported and ones that are not and stitch the modules together in the deployment application or you can register converters for missing ops.
       </p>
       <blockquote>
        <div>
         <p>
          You can check support without going through the full compilation pipleine using the
          <code class="docutils literal notranslate">
           <span class="pre">
            trtorch::CheckMethodOperatorSupport(const
           </span>
           <span class="pre">
            torch::jit::Module&amp;
           </span>
           <span class="pre">
            module,
           </span>
           <span class="pre">
            std::string
           </span>
           <span class="pre">
            method_name)
           </span>
          </code>
          api
to see what operators are not supported.
         </p>
        </div>
       </blockquote>
       <span id="custom-converters">
       </span>
       <h3 id="registering-custom-converters">
        Registering Custom Converters
        <a class="headerlink" href="#registering-custom-converters" title="Permalink to this headline">
         ¶
        </a>
       </h3>
       <p>
        Operations are mapped to TensorRT through the use of modular converters, a function that takes a node from a the JIT graph and produces an equivalent layer or subgraph in TensorRT.
TRTorch ships with a library of these converters stored in a registry, that will be executed depending on the node being parsed. For instance a
        <code class="docutils literal notranslate">
         <span class="pre">
          aten::relu(%input0.4)
         </span>
        </code>
        instruction will trigger
the relu converter to be run on it, producing an activation layer in the TensorRT graph. But since this library is not exhaustive you may need to write your own to get TRTorch
to support your module.
       </p>
       <p>
        Shipped with the TRTorch distribution are the internal core API headers. You can therefore access the converter registry and add a converter for the op you need.
       </p>
       <p>
        For example, if we try to compile a graph with a build of TRTorch that doesn’t support the flatten operation (
        <code class="docutils literal notranslate">
         <span class="pre">
          aten::flatten
         </span>
        </code>
        ) you may see this error:
       </p>
       <div class="highlight-none notranslate">
        <div class="highlight">
         <pre><span></span>terminate called after throwing an instance of 'trtorch::Error'
what():  [enforce fail at core/conversion/conversion.cpp:109] Expected converter to be true but got false
Unable to convert node: %input.1 : Tensor = aten::flatten(%x.1, %11, %5) # x.py:25:0 (conversion.AddLayer)
Schema: aten::flatten.using_ints(Tensor self, int start_dim=0, int end_dim=-1) -&gt; (Tensor)
Converter for aten::flatten requested, but no such converter was found.
If you need a converter for this operator, you can try implementing one yourself
or request a converter: https://www.github.com/NVIDIA/TRTorch/issues
</pre>
        </div>
       </div>
       <p>
        We can register a converter for this operator in our application. All of the tools required to build a converter can be imported by including
        <code class="docutils literal notranslate">
         <span class="pre">
          trtorch/core/conversion/converters/converters.h
         </span>
        </code>
        .
We start by creating an instance of the self-registering class
        <code class="docutils literal notranslate">
         <span class="pre">
          trtorch::core::conversion::converters::RegisterNodeConversionPatterns()
         </span>
        </code>
        which will register converters
in the global converter registry, associating a function schema like
        <code class="docutils literal notranslate">
         <span class="pre">
          aten::flatten.using_ints(Tensor
         </span>
         <span class="pre">
          self,
         </span>
         <span class="pre">
          int
         </span>
         <span class="pre">
          start_dim=0,
         </span>
         <span class="pre">
          int
         </span>
         <span class="pre">
          end_dim=-1)
         </span>
         <span class="pre">
          -&gt;
         </span>
         <span class="pre">
          (Tensor)
         </span>
        </code>
        with a lambda that
will take the state of the conversion, the node/operation in question to convert and all of the inputs to the node and produces as a side effect a new layer in the TensorRT network.
Arguments are passed as a vector of inspectable unions of TensorRT
        <code class="docutils literal notranslate">
         <span class="pre">
          ITensors
         </span>
        </code>
        and Torch
        <code class="docutils literal notranslate">
         <span class="pre">
          IValues
         </span>
        </code>
        in the order arguments are listed in the schema.
       </p>
       <p>
        Below is a implementation of a
        <code class="docutils literal notranslate">
         <span class="pre">
          aten::flatten
         </span>
        </code>
        converter that we can use in our application. You have full access to the Torch and TensorRT libraries in the converter implementation. So
for example we can quickly get the output size by just running the operation in PyTorch instead of implementing the full calculation outself like we do below for this flatten converter.
       </p>
       <div class="highlight-c++ notranslate">
        <div class="highlight">
         <pre><span></span><span class="cp">#include</span> <span class="cpf">"torch/script.h"</span><span class="cp"></span>
<span class="cp">#include</span> <span class="cpf">"trtorch/trtorch.h"</span><span class="cp"></span>
<span class="cp">#include</span> <span class="cpf">"trtorch/core/conversion/converters/converters.h"</span><span class="cp"></span>

<span class="k">static</span> <span class="k">auto</span> <span class="n">flatten_converter</span> <span class="o">=</span> <span class="n">trtorch</span><span class="o">::</span><span class="n">core</span><span class="o">::</span><span class="n">conversion</span><span class="o">::</span><span class="n">converters</span><span class="o">::</span><span class="n">RegisterNodeConversionPatterns</span><span class="p">()</span>
    <span class="p">.</span><span class="n">pattern</span><span class="p">({</span>
        <span class="s">"aten::flatten.using_ints(Tensor self, int start_dim=0, int end_dim=-1) -&gt; (Tensor)"</span><span class="p">,</span>
        <span class="p">[](</span><span class="n">trtorch</span><span class="o">::</span><span class="n">core</span><span class="o">::</span><span class="n">conversion</span><span class="o">::</span><span class="n">ConversionCtx</span><span class="o">*</span> <span class="n">ctx</span><span class="p">,</span>
           <span class="k">const</span> <span class="n">torch</span><span class="o">::</span><span class="n">jit</span><span class="o">::</span><span class="n">Node</span><span class="o">*</span> <span class="n">n</span><span class="p">,</span>
           <span class="n">trtorch</span><span class="o">::</span><span class="n">core</span><span class="o">::</span><span class="n">conversion</span><span class="o">::</span><span class="n">converters</span><span class="o">::</span><span class="n">args</span><span class="o">&amp;</span> <span class="n">args</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kt">bool</span> <span class="p">{</span>
            <span class="k">auto</span> <span class="n">in</span> <span class="o">=</span> <span class="n">args</span><span class="p">[</span><span class="mi">0</span><span class="p">].</span><span class="n">ITensor</span><span class="p">();</span>
            <span class="k">auto</span> <span class="n">start_dim</span> <span class="o">=</span> <span class="n">args</span><span class="p">[</span><span class="mi">1</span><span class="p">].</span><span class="n">unwrapToInt</span><span class="p">();</span>
            <span class="k">auto</span> <span class="n">end_dim</span> <span class="o">=</span> <span class="n">args</span><span class="p">[</span><span class="mi">2</span><span class="p">].</span><span class="n">unwrapToInt</span><span class="p">();</span>
            <span class="k">auto</span> <span class="n">in_shape</span> <span class="o">=</span> <span class="n">trtorch</span><span class="o">::</span><span class="n">core</span><span class="o">::</span><span class="n">util</span><span class="o">::</span><span class="n">toVec</span><span class="p">(</span><span class="n">in</span><span class="o">-&gt;</span><span class="n">getDimensions</span><span class="p">());</span>
            <span class="k">auto</span> <span class="n">out_shape</span> <span class="o">=</span> <span class="n">torch</span><span class="o">::</span><span class="n">flatten</span><span class="p">(</span><span class="n">torch</span><span class="o">::</span><span class="n">rand</span><span class="p">(</span><span class="n">in_shape</span><span class="p">),</span> <span class="n">start_dim</span><span class="p">,</span> <span class="n">end_dim</span><span class="p">).</span><span class="n">sizes</span><span class="p">();</span>

            <span class="k">auto</span> <span class="n">shuffle</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">-&gt;</span><span class="n">net</span><span class="o">-&gt;</span><span class="n">addShuffle</span><span class="p">(</span><span class="o">*</span><span class="n">in</span><span class="p">);</span>
            <span class="n">shuffle</span><span class="o">-&gt;</span><span class="n">setReshapeDimensions</span><span class="p">(</span><span class="n">trtorch</span><span class="o">::</span><span class="n">core</span><span class="o">::</span><span class="n">util</span><span class="o">::</span><span class="n">toDims</span><span class="p">(</span><span class="n">out_shape</span><span class="p">));</span>
            <span class="n">shuffle</span><span class="o">-&gt;</span><span class="n">setName</span><span class="p">(</span><span class="n">trtorch</span><span class="o">::</span><span class="n">core</span><span class="o">::</span><span class="n">util</span><span class="o">::</span><span class="n">node_info</span><span class="p">(</span><span class="n">n</span><span class="p">).</span><span class="n">c_str</span><span class="p">());</span>

            <span class="k">auto</span> <span class="n">out_tensor</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">-&gt;</span><span class="n">AssociateValueAndTensor</span><span class="p">(</span><span class="n">n</span><span class="o">-&gt;</span><span class="n">outputs</span><span class="p">()[</span><span class="mi">0</span><span class="p">],</span> <span class="n">shuffle</span><span class="o">-&gt;</span><span class="n">getOutput</span><span class="p">(</span><span class="mi">0</span><span class="p">));</span>
            <span class="k">return</span> <span class="nb">true</span><span class="p">;</span>
        <span class="p">}</span>
    <span class="p">});</span>

<span class="kt">int</span> <span class="nf">main</span><span class="p">()</span> <span class="p">{</span>
    <span class="p">...</span>
</pre>
        </div>
       </div>
       <p>
        You can find more information on all the details of writing converters in the contributors documentation (
        <a class="reference internal" href="../contributors/writing_converters.html#writing-converters">
         <span class="std std-ref">
          Writing Converters
         </span>
        </a>
        ).
If you find yourself with a large library of converter implementations, do consider upstreaming them, PRs are welcome and it would be great for the community to benefit as well.
       </p>
      </article>
     </div>
    </div>
   </main>
  </div>
  <footer class="md-footer">
   <div class="md-footer-nav">
    <nav class="md-footer-nav__inner md-grid">
     <a class="md-flex md-footer-nav__link md-footer-nav__link--prev" href="installation.html" rel="prev" title="Installation">
      <div class="md-flex__cell md-flex__cell--shrink">
       <i class="md-icon md-icon--arrow-back md-footer-nav__button">
       </i>
      </div>
      <div class="md-flex__cell md-flex__cell--stretch md-footer-nav__title">
       <span class="md-flex__ellipsis">
        <span class="md-footer-nav__direction">
         Previous
        </span>
        Installation
       </span>
      </div>
     </a>
     <a class="md-flex md-footer-nav__link md-footer-nav__link--next" href="ptq.html" rel="next" title="Post Training Quantization (PTQ)">
      <div class="md-flex__cell md-flex__cell--stretch md-footer-nav__title">
       <span class="md-flex__ellipsis">
        <span class="md-footer-nav__direction">
         Next
        </span>
        Post Training Quantization (PTQ)
       </span>
      </div>
      <div class="md-flex__cell md-flex__cell--shrink">
       <i class="md-icon md-icon--arrow-forward md-footer-nav__button">
       </i>
      </div>
     </a>
    </nav>
   </div>
   <div class="md-footer-meta md-typeset">
    <div class="md-footer-meta__inner md-grid">
     <div class="md-footer-copyright">
      <div class="md-footer-copyright__highlight">
       © Copyright 2020, NVIDIA Corporation.
      </div>
      Created using
      <a href="http://www.sphinx-doc.org/">
       Sphinx
      </a>
      3.0.3.
             and
      <a href="https://github.com/bashtage/sphinx-material/">
       Material for
              Sphinx
      </a>
     </div>
    </div>
   </div>
  </footer>
  <script src="../_static/javascripts/application.js">
  </script>
  <script>
   app.initialize({version: "1.0.4", url: {base: ".."}})
  </script>
 </body>
</html>